# -*- coding: utf-8 -*-
"""
Created on 2024-04-25 12:25:26
---------
@summary:
---------
@author: Alex
"""
from venv import logger
from items.amazon_item_detail import AmazonDetailItem
from items.amazon_item_comment import AmazonCommentItem
from requests.models import Response as RResponse
import feapder
from spiders.amazon.amazon_spider_base import BaseSpider
from feapder.utils import tools
from copy import deepcopy


class AmazonDetailSpider(BaseSpider):

    def start_requests(self, task):
        id, asin, cid = task  # id， url为所取的字段，main函数中指定的
        url = f"{self.host}/dp/{asin}/?th=1"
        yield feapder.Request(url=url, task_id=id, asin=asin, cid=cid)

    def get_category(self, response):
        cate_info = []
        _category_list = response.css(
            "#wayfinding-breadcrumbs_feature_div .a-unordered-list.a-horizontal  li")
        k = -1
        for _category in _category_list or []:
            name = _category.css("a::text").extract_first()
            if name:
                k += 1
                url = _category.css("a::attr(href)").extract_first()
                cate_info.append(
                    {"index": k, "url": url, "name": name.strip()})
        return cate_info

    def get_cover_list(self, response, asin, cid):  # 因为价格不全，只负责解析自己的信息 和新收录asin

        data_str = response.re_first(
            "obj = jQuery.parseJSON\('(.*?)'\);\ndata\[")
        if not data_str:  # 特殊页面 B00DTOX9A2
            return   {"imgs_info": [{"index": 0, "url": response.css("#js-masrw-main-image ::attr(src)").extract_first(), "variant": ''}], "videos_info": [],
               "parentAsin": asin}
        data = {}
        for i in range(2):
            try:
                data = tools.json.loads(data_str)
                break
            except Exception as e:
                print("json format fail")
                data_str = data_str.replace('\\', '\\\\')
        if not data:
            print("data_str+==", data_str)

        landingAsinColor = data['landingAsinColor']  # 当前型号
        if 'Remote only' not in data['colorToAsin']:
            for _type in data['colorToAsin']:
                _asin = data['colorToAsin'][_type]['asin']
                if asin != _asin:
                    self.insert_asin(_asin, cid)   # 新收录asin
        imgs_info = []
        # print ("colorImages",data['colorImages'])
        if data.get('colorImages'):
            for index, i in enumerate(data['colorImages'].get(landingAsinColor, [])):
                img_url = i.get("hiRes", i.get(
                    "large", i.get("thumb")))  # 高清图--大图-小图
                imgs_info.append(
                    {"index": index, "url": img_url, "variant": i['variant']})
        else:
            imgs_info.append({"index": "0", "url": response.css(
                '#landingImage ::attr(data-old-hires)').extract_first(), "variant": landingAsinColor})  # 只有一张图的情况:B01I0IGFKC
        videos_info = []
        _videos = data.get("videos")
        for index, v in enumerate(_videos or []):
            videos_info.append({"index": index, "url": v['url'], "holderId": v['holderId'], "durationTimestamp": v[
                               'durationTimestamp'], "slateUrl": v['slateUrl'], "variant": v['variant'], "title": v['title']})

        ret = {"imgs_info": imgs_info, "videos_info": videos_info,
               "parentAsin": data['parentAsin']}
        return ret

    def get_rank_info(self, response):
        ret = None
        elem = response.css("#zeitgeistBadge_feature_div a")
        if elem:
            ret = {"url": elem.css("::attr(href)").extract_first(), "category": elem.css(
                "::attr(title)").extract_first(), "rank": elem.css("i::text").extract_first()}
        return ret

    def get_current_asinVariationValues(self, response, asin, currentAsin):
        _datas = response.re_first('asinVariationValues" : (\{.*?),\n')
        if _datas:
            data = tools.json.loads(_datas)
            return data.get(asin, data.get(currentAsin))

    def get_feature_info(self, response):  # x
        info = response.css("#productOverview_feature_div tr")
        ret = []
        for elem in (info or []):
            td1 = elem.css("td:nth-of-type(1) span::text").extract_first()
            if td1.strip():
                ret.append(
                    {td1: elem.css("td:nth-of-type(2) span::text").extract_first().strip()})
        if not ret:
            feature_eme = response.css("#mas-product-feature")
            if feature_eme:
                ret.append({feature_eme.css(" h2 ::text").extract_first().strip():[i.strip() for i in feature_eme.css(" li ::text").extract() if i.strip() ]})
        return ret

    def get_videos_info(self, response):  # x
        videos_info = []

        for index, elem in enumerate(response.css("#vse-vw-dp-vse-related-videos li div[data-product-asin]") or []):
            videos_info.append({"index": index,
                                "url": elem.css("::attr(data-video-url)").extract_first(),
                                "asin": elem.css("::attr(data-product-asin)").extract_first(),
                                "holderId": elem.css("::attr(data-vendor-name)").extract_first(),
                                "durationTimestamp": elem.css("::attr(data-formatted-duration)").extract_first(),
                                "slateUrl": elem.css("::attr(data-video-image-url)").extract_first(),
                                "data-profile-image-url": elem.css(".a-profile-avatar img::attr(data-src)").extract_first(),
                                "title": elem.css(" h4._vse-vw-dp-card_style_vseVideoTitleText__1lJWI ::text").extract_first()
                                })
        return videos_info

    def get_product_info(self, response):  # x
        info = response.css("#prodDetails tr")
        ret = []
        for elem in (info or []):

            ret.append({elem.css("th ::text").extract_first()
                       : elem.css("td ::text").extract_first()})
        if not ret:
            feature_text = response.css('h2:contains("Product Details") ~ .a-section.a-spacing-small ::text').extract()
            info = []
            for t in feature_text or []:                
                if 'Customer reviews' in t:
                    break
                info.append(t)
            if info:
                ret.append({"Product Details":info})
        
        return ret

    def get_customer_tag(self, response):
        product_summary = response.css(
            "#product-summary span ::text").extract_first()
        tag_list = response.css(
            "#cr-product-insights-cards button ::text").extract()
        ai_cmts = []
        for index, i in enumerate(tag_list):
            ai_item = {"index": index, "tag": i, "cmt_item": []}
            _css = f"#aspect-bottom-sheet-0-{index} .a-section"
            k = -1
            for m_ele in response.css(_css):
                rate_info = m_ele.css(
                    '._cr-product-insights_style_stat-text-box-label-row__2ZAPB ::text').extract()
                if rate_info:
                    ai_item['rate_info'] = rate_info  # 直接列表
                url = m_ele.css("p a::attr(href)").extract_first()
                if url:
                    k += 1
                    cmt_item = {
                        "index": k,
                        "content": ".".join(m_ele.css("p *::text").extract()),
                        "ai_hit": m_ele.css("p b::text").extract(),
                        "type": m_ele.css("p a ::attr(data-csa-c-element-type)").extract_first(),
                        "url": url
                    }
                    ai_item['cmt_item'].append(cmt_item)

            ai_cmts.append(ai_item)
        return {"product_summary": product_summary, "tag_list": tag_list, "ai_cmt": ai_cmts}

    def parse(self, request, response):
        task_id = request.task_id
        _title = response.css('#productTitle ::text').extract_first() \
            or response.css('title ::text').extract_first() \
            or ''.join([i for i in response.css('#btAsinTitle ::text').extract() if i.strip() ])  or ''
        
        title = _title
        cid = request.cid
        asin = request.asin
        print('asin++', asin)
        if not title:
            raise Exception("无效 title html")
        item = AmazonDetailItem()
        item.asin = asin
        item.url = request.url
        item.title = title
        item.cids = cid
        item.cate_info = self.get_category(response)
        cover_list = self.get_cover_list(response, asin, cid)
        item.parent_asin = cover_list.pop("parentAsin")
        item.cover = cover_list['imgs_info'][0]['url']
        if not item.cover:
            raise Exception(f"未能识别的页面:{request.url}")
        item.cover_list = cover_list
        shop_elem =  response.css('#bylineInfo') or  response.css('#brand')
        item.shop_name =shop_elem.css("::text").extract_first() 
        item.shop_url = shop_elem.css('::attr(href)').extract_first()
        item.score_info = response.css(
            query='#averageCustomerReviews .a-size-base.a-color-base ::text').extract_first()
        item.rank_info = self.get_rank_info(response)
        item.sold_info = response.css(
            "#social-proofing-faceout-title-tk_bought span ::text").extract_first()
        _price = response.css('.a-section.a-spacing-none.aok-align-center.aok-relative .aok-offscreen ::text').extract_first() or \
            response.css(
                '#corePrice_desktop .a-offscreen ::text').extract_first() \
            or "".join([i for i in response.css("#actualPriceValue *::text").extract()])

        if _price:
            _price = _price.replace('\n','').strip()
        item.price_info = _price
        currentAsin = response.re_first(
            regex='currentPageAsin":[\s\S]*?"(\w+)",')
        item.sku_info = self.get_current_asinVariationValues(
            response, asin, currentAsin)  # B0BWKRN7CZ #B07F6NLZSS没有
        item.about_info = "\n".join([i for i in response.css(
            "#feature-bullets *::text").extract() if i.strip()])
        item.feature_info = self.get_feature_info(response)
        item.videos_info = self.get_videos_info(response)
        item.product_info = self.get_product_info(response)
        item.other_seller_info = None
        item.comment_num = response.css(
            query=".a-row.a-spacing-medium.averageStarRatingNumerical ::text").extract_first()
        item.customer_tag = self.get_customer_tag(response)
        yield item
        if currentAsin and currentAsin != asin:
            item2 = deepcopy(item)
            item2['asin'] = currentAsin
            yield item2

        # top评论
        for cmt_item in self.parse_cmt_top_div(response, item.parent_asin):
            yield cmt_item

        if item.comment_num:
            cmt_recent_url = "https://www.amazon.com/hz/reviews-render/ajax/medley-filtered-reviews/get/ref=cm_cr_dp_d_fltrs_srt"
            data = {"language": "en_US",
                    "asin": asin,
                    "sortBy": "recent",
                    "scope": "reviewsAjax1",
                    }
            # 最近的评论
            yield feapder.Request(url=cmt_recent_url, method="POST", data=data, callback=self.parse_cmt_recent_div, asin=item.parent_asin, task_id=task_id)

        yield self.update_task_batch(request.task_id, 1, asin=asin)  # 更新任务状态为1

    def parse_cmt_recent_div(self, request, response):
        asin = request.asin
        rtype = "most_recent"
        for data_str in response.text.split("&&&"):
            data_str = data_str.strip()
            if not data_str:
                continue
            data_list = None
            for i in range(2):
                try:
                    data_list = tools.json.loads(data_str)
                    break
                except Exception as e:
                    data_list = data_str.split(",")
            if not data_list:
                logger.error(f"cmt div 错误:{asin}")
                continue
            for content in data_list:
                if content and content.endswith("]"):
                    content = content[:-1]
                if "a-section review aok-relative" in content:
                    tmp_response = RResponse()
                    tmp_response.status_code = 200
                    # tmp_response._content = content
                    tmp_response.headers['Content-Type'] = 'text/html;charset=UTF-8'
                    tmp_response.encoding = "utf-8"
                    tmp_response.__dict__['_content'] = content.encode()
                    tmp_response = feapder.Response(tmp_response, False)
                    item = self._parse_cmt_div(
                        cmt_css=tmp_response, asin=asin, rtype=rtype)
                    if item:
                        yield item

    def _parse_cmt_div(self, cmt_css, asin, rtype):
        comment_date_text = cmt_css.css(
            "span[data-hook=review-date] ::text").extract_first()
        if not comment_date_text:
            return
        item = AmazonCommentItem()
        item.asin = asin
        author_eme = cmt_css.css(".a-profile")
        item.author_cover = author_eme.css("img ::attr(src)").extract_first()
        item.author_name = author_eme.css(
            ".a-profile-name ::text").extract_first()
        item.author_url = author_eme.css("::attr(href)").extract_first()
        item.comment_date_text = comment_date_text
        item.content = "".join(cmt_css.css(
            "div[data-hook=review-collapsed] span::text").extract())
        item.help_info = cmt_css.css(
            "span[data-hook=helpful-vote-statement] ::text").extract_first()
        item.media_list = cmt_css.css(
            ".review-image-tile-section img::attr(data-src)").extract()
        item.rtype = rtype  # 评论来源类型:ai,top_review,most_recent
        item.score = cmt_css.css(
            "i[data-hook=review-star-rating] ::text").extract_first()
        item.sku_info = cmt_css.css(
            "span[data-hook=format-strip-linkless] ::text").extract_first()
        title_eme = cmt_css.css("a[data-hook=review-title] ")
        item.title = title_eme.css("span ::text").extract()[-1]
        url = title_eme.css("::attr(href)").extract_first()
        if "https:" not in url:
            url = self.host+url
        item.url = url

        item.cmt_id = cmt_css.css("div[data-reviewid] ::attr(data-reviewid)").extract_first(
        ) or tools.re.findall("customer-reviews/(\w+)[\/]?", item.url)[0]
        return item

    def parse_cmt_top_div(self, response, asin):
        rtype = "top_review"
        cmts_css = response.css('#cm-cr-dp-review-list div[data-hook]')
        for cmt_css in cmts_css:
            item = self._parse_cmt_div(cmt_css, asin, rtype)
            if item:
                yield item


if __name__ == "__main__":
    AmazonDetailSpider().start()
